'http://www.gov.cn/xinwen/yaowen.htm'
'中央政府网'

import requests,json,os,time
from datetime import datetime
from lxml import etree
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess,productPrint
SPIDERNAME='国务院'

def parseGovArtPubTime(time_str):
    #2020-06-13
    return time_str+' 00:00:00'
                     

def parseGovArtContent(url):
    r=sess.get(url)
    r.encoding='utf8'
    selector=etree.HTML(r.text)
    eleList=selector.cssselect('#UCAP-CONTENT')
    if eleList:
        ele=eleList[0]
        content=ele.xpath('string(.)').strip().strip('\n')
    else:
        print('网页内容为空',url)
        content=''

    return content

def getGovArticleList(articleCol,BeCrawledUrlList):
    url_1='http://www.gov.cn/xinwen/yaowen.htm'  #新闻
    url_2='http://www.gov.cn/zhengce/zuixin.htm' #政策
    temp_article_ls=[]
    for url in (url_1,url_2):
        r=sess.get(url)
        r.encoding='utf8'
        selector=etree.HTML(r.text)
        eleList=selector.xpath("//div[@class='list list_1 list_2']/ul/li[not(@class)]")
        for ele in  eleList:
            articleUrl=ele.xpath('./h4/a/@href')[0]
            if 'http://www.gov.cn' not in articleUrl:
                articleUrl='http://www.gov.cn'+articleUrl
            #判断是否已经爬取过，如果是，跳出循环
            if len(articleCol.search(where('url')==articleUrl)):break   #有置顶项目
            # print(articleUrl)
            try:
                title=ele.xpath('./h4/a/text()')[0]
            except IndexError as e: 
                print(e)
                continue
            publicTime=parseGovArtPubTime(ele.xpath('./h4/span/text()')[0].strip())
            temp_dict={'tags':['gov'],'score':0,'uid':UID()}
            temp_dict['title']=title.strip()
            temp_dict['articleFrom']='gov.cn'
            temp_dict['url']=articleUrl.strip()
            temp_dict['publicTime']=publicTime.strip()
            #文章内容
            content=parseGovArtContent(articleUrl)
            temp_dict['content']=content
            #定文章所属期货品种,板块
            n=parseContentToName(title+content)
            temp_dict=productPrint(n,SPIDERNAME,title,temp_dict,ProductToGroup)
            temp_article_ls.append(temp_dict)

    #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)


